{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|  Name   |     City      |   City2 |\n",
      "|:-------:|:-------------:|--------:|\n",
      "|  Alice  |   New York    |       2 |\n",
      "|   Bob   | San Francisco |         |\n",
      "| Charlie |    Seattle    |         |\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "data = [\n",
    "    {\"Name\": \"Alice\", \"City\": \"New York\", \"City2\": 2},\n",
    "    {\"Name\": \"Bob\", \"City\": \"San Francisco\"},\n",
    "    {\"Name\": \"Charlie\", \"City\": \"Seattle\"}\n",
    "]\n",
    "\n",
    "table = tabulate(data, headers=\"keys\", tablefmt=\"pipe\", stralign='center')\n",
    "\n",
    "print(table)\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-25T14:37:39.908082600Z",
     "start_time": "2023-12-25T14:37:39.906118400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- apple banana cherry date\n",
      "+ apple cherry date elderberry\n"
     ]
    }
   ],
   "source": [
    "import difflib\n",
    "\n",
    "text1 = \"apple banana cherry date\"\n",
    "text2 = \"apple cherry date elderberry\"\n",
    "\n",
    "# 创建 Differ 对象\n",
    "d = difflib.Differ()\n",
    "\n",
    "# 通过 Differ 对象的 compare 方法比较两个文本\n",
    "diff = d.compare(text1.split(\"\\n\"), text2.split(\"\\n\"))\n",
    "\n",
    "# 将比较结果转换为字符串并打印出来\n",
    "print('\\n'.join(diff))\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-25T15:03:24.902182200Z",
     "start_time": "2023-12-25T15:03:24.898740800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['statistic', 'age', 'money'], ['sum', 150.0, 130.0], ['mean', 30.0, 26.0], ['median', 30.0, 23.6]]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# 原始数据\n",
    "list_of_dicts = [\n",
    "    {\"name\": \"Alice\", \"age\": 25, \"city\": \"New York\", \"money\": 35.6},\n",
    "    {\"name\": \"Bob\", \"age\": 30, \"city\": \"San Francisco\", \"money\": 23.6},\n",
    "    {\"name\": \"Charlie\", \"age\": 35, \"city\": \"Seattle\", \"money\": 13.6},\n",
    "    {\"name\": \"David\", \"age\": 28, \"city\": \"New York\", \"money\": 53.6},\n",
    "    {\"name\": \"Eve\", \"age\": 32, \"city\": \"San Francisco\", \"money\": 3.6}\n",
    "]\n",
    "\n",
    "# 转为Dataframe\n",
    "df = pd.DataFrame(list_of_dicts)\n",
    "\n",
    "# 识别包含数字的列，对其进行统计\n",
    "numeric_columns = df.select_dtypes(include=['int', 'float']).columns\n",
    "statistics = df[numeric_columns].agg(['sum', 'mean', 'median']).reset_index()\n",
    "\n",
    "# 将结果保存至一个List[list]\n",
    "result = statistics.values.tolist()\n",
    "result.insert(0, ['statistic', *numeric_columns])\n",
    "\n",
    "print(result)\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-26T07:42:03.299172100Z",
     "start_time": "2023-12-26T07:42:03.295054800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'D:\\\\IDEA_Projects\\\\TestJavaCode\\\\src\\\\main\\\\java\\\\org\\\\apache\\\\commons\\\\lang3\\\\Example.java'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mFileNotFoundError\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[60], line 48\u001B[0m\n\u001B[0;32m     43\u001B[0m             method_signatures\u001B[38;5;241m.\u001B[39mappend(method_signature)\n\u001B[0;32m     45\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m method_signatures\n\u001B[1;32m---> 48\u001B[0m java_code \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mopen\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;124;43mr\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mD:\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mIDEA_Projects\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mTestJavaCode\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43msrc\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mmain\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mjava\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43morg\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mapache\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mcommons\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mlang3\u001B[39;49m\u001B[38;5;124;43m\\\u001B[39;49m\u001B[38;5;124;43mExample.java\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39mread()\n\u001B[0;32m     49\u001B[0m \u001B[38;5;66;03m# 获取方法签名\u001B[39;00m\n\u001B[0;32m     50\u001B[0m signatures \u001B[38;5;241m=\u001B[39m get_method_signatures(java_code)\n",
      "File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:310\u001B[0m, in \u001B[0;36m_modified_open\u001B[1;34m(file, *args, **kwargs)\u001B[0m\n\u001B[0;32m    303\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m file \u001B[38;5;129;01min\u001B[39;00m {\u001B[38;5;241m0\u001B[39m, \u001B[38;5;241m1\u001B[39m, \u001B[38;5;241m2\u001B[39m}:\n\u001B[0;32m    304\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[0;32m    305\u001B[0m         \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mIPython won\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt let you open fd=\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mfile\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m by default \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    306\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    307\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124myou can use builtins\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m open.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    308\u001B[0m     )\n\u001B[1;32m--> 310\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mio_open\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfile\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[1;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: 'D:\\\\IDEA_Projects\\\\TestJavaCode\\\\src\\\\main\\\\java\\\\org\\\\apache\\\\commons\\\\lang3\\\\Example.java'"
     ]
    }
   ],
   "source": [
    "import javalang\n",
    "\n",
    "\n",
    "def get_type_string(type):\n",
    "    if type is None:\n",
    "        return 'void'\n",
    "    # 处理基本类型和引用类型\n",
    "    type_str = ''\n",
    "    if isinstance(type, javalang.tree.ReferenceType):\n",
    "        type_str = type.name\n",
    "        if type.arguments:\n",
    "            args = ', '.join(get_type_string(arg.type) for arg in type.arguments if arg.type is not None)\n",
    "            type_str += f\"<{args}>\"\n",
    "    elif isinstance(type, javalang.tree.BasicType):\n",
    "        type_str = type.name\n",
    "    else:\n",
    "        type_str = str(type)\n",
    "\n",
    "    # 处理数组类型\n",
    "    dimensions = ''.join('[]' for _ in range(len(type.dimensions))) if hasattr(type, 'dimensions') else ''\n",
    "    return f\"{type_str}{dimensions}\"\n",
    "\n",
    "\n",
    "def get_method_signatures(java_code):\n",
    "    tree = javalang.parse.parse(java_code)\n",
    "    method_signatures = []\n",
    "\n",
    "    for _, class_declaration in tree.filter(javalang.tree.ClassDeclaration):\n",
    "        for method in class_declaration.methods:\n",
    "            # 获取方法的返回类型，包括泛型信息和数组\n",
    "            return_type = get_type_string(method.return_type)\n",
    "\n",
    "            # 获取方法的参数类型，包括泛型信息和数组\n",
    "            params = []\n",
    "            for parameter in method.parameters:\n",
    "                param_type = get_type_string(parameter.type)\n",
    "                if parameter.varargs:\n",
    "                    param_type += \"...\"\n",
    "                params.append(param_type)\n",
    "\n",
    "            # 构建方法签名\n",
    "            method_signature = f\"{method.name}({', '.join(params)}) : {return_type}\"\n",
    "            method_signatures.append(method_signature)\n",
    "\n",
    "    return method_signatures\n",
    "\n",
    "\n",
    "java_code = open(r\"D:\\IDEA_Projects\\TestJavaCode\\src\\main\\java\\org\\apache\\commons\\lang3\\Example.java\").read()\n",
    "# 获取方法签名\n",
    "signatures = get_method_signatures(java_code)\n",
    "print(signatures)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-24T09:03:46.368543100Z",
     "start_time": "2024-01-24T09:03:46.221276500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "ArrayUtils\n",
      "StringUtils\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "import javalang\n",
    "from javalang.tree import ClassDeclaration, MethodDeclaration\n",
    "\n",
    "method_filter = ['Hashcode']\n",
    "\n",
    "from typing import List\n",
    "\n",
    "\n",
    "\n",
    "def compress_data(content: str, signature: str, method_infos: List[dict]) -> str:\n",
    "    # 将内容按行分割\n",
    "    lines = content.split('\\n')\n",
    "\n",
    "    # 为了不影响行号，我们需要从后往前替换\n",
    "    # 首先对method_infos按照start从大到小排序\n",
    "    method_infos_sorted = sorted(method_infos, key=lambda x: -x['start'])\n",
    "\n",
    "    # 遍历每个方法信息\n",
    "    for method_info in method_infos_sorted:\n",
    "        # 如果签名不匹配，则压缩方法体\n",
    "        if method_info['signature'] != signature:\n",
    "            # 注意，行号从1开始，列表索引从0开始，所以要减1\n",
    "            start_index = method_info['start'] - 1\n",
    "            end_index = method_info['end']\n",
    "            # 将方法体替换为方法签名\n",
    "            lines[start_index:end_index] = [method_info['signature']]\n",
    "\n",
    "    # 将修改后的行列表重新组合成一个字符串\n",
    "    compressed_content = '\\n'.join(lines)\n",
    "    return compressed_content\n",
    "\n",
    "\n",
    "\n",
    "def remove_comments(input_string):\n",
    "    import re\n",
    "    pattern = r'/\\*\\*?\\s.*?\\*/'\n",
    "    return re.sub(pattern, '', input_string, flags=re.DOTALL)\n",
    "\n",
    "\n",
    "def get_class_info(file: str):\n",
    "    file = remove_comments(file)\n",
    "\n",
    "    file_lines = file.splitlines()\n",
    "    try:\n",
    "        tree = javalang.parse.parse(file)\n",
    "    except Exception as e:\n",
    "        print(file)\n",
    "        raise e\n",
    "    assert len(tree.types) == 1  # 一个文件只有一个类\n",
    "    clazz: ClassDeclaration = tree.types[0]\n",
    "    if not isinstance(clazz, ClassDeclaration) or 'abstract' in clazz.modifiers:\n",
    "        return None, None, None\n",
    "    clazz_name = clazz.name\n",
    "    methods: List[MethodDeclaration] = clazz.methods\n",
    "\n",
    "    method_infos = []\n",
    "    all_method_infos = []\n",
    "    for method in methods:\n",
    "        valid_focal_method =  'public' in method.modifiers\n",
    "\n",
    "        start = method.position[0]\n",
    "        end = start\n",
    "        stack1 = []\n",
    "        flag = False\n",
    "        while not flag or len(stack1) != 0:\n",
    "            line = file_lines[end - 1]\n",
    "            for ch in line:\n",
    "                if ch == \"{\":\n",
    "                    flag = True\n",
    "                    stack1.append(ch)\n",
    "                elif ch == \"}\":\n",
    "                    stack1.pop()\n",
    "            end += 1\n",
    "            if end > 10000:\n",
    "                assert False\n",
    "        content = file_lines[start - 1:end]\n",
    "        # 缩进处理\n",
    "        indent = len(content[0]) - len(content[0].lstrip())\n",
    "        content = [line[indent:] for line in content]\n",
    "        content = \"\\n\".join(content)\n",
    "\n",
    "        # 获取方法的返回类型，包括泛型信息和数组\n",
    "        return_type = get_type_string(method.return_type)\n",
    "\n",
    "        # 获取方法的参数类型，包括泛型信息和数组\n",
    "        params = []\n",
    "        for parameter in method.parameters:\n",
    "            param_type = get_type_string(parameter.type)\n",
    "            if parameter.varargs:\n",
    "                param_type += \"...\"\n",
    "            params.append(param_type)\n",
    "\n",
    "        # 构建方法签名\n",
    "        method_signature = f\"{method.name}({', '.join(params)}) : {return_type}\"\n",
    "        info = {\n",
    "            \"signature\": method_signature,\n",
    "            \"name\": method.name,\n",
    "            \"start\": start,\n",
    "            \"end\": end,\n",
    "            \"content\": content,\n",
    "        }\n",
    "        if valid_focal_method:\n",
    "            method_infos.append(info)\n",
    "        all_method_infos.append(info)\n",
    "    \n",
    "    if clazz_name == 'ArrayUtils':\n",
    "        print(1)\n",
    "    for mi in method_infos:\n",
    "        mi[\"compressed_content\"] = compress_data(file, mi[\"signature\"], all_method_infos)\n",
    "            \n",
    "    return clazz_name, file, method_infos\n",
    "\n",
    "# get_class_info(open(r\"D:\\IDEA_Projects\\TestJavaCode\\src\\main\\java\\net\\mooctest\\Queue.java\").read())[2]\n",
    "import zipfile\n",
    "import tiktoken\n",
    "import os\n",
    "from io import BytesIO\n",
    "\n",
    "tiktoken_encoder = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n",
    "result_list = []\n",
    "\n",
    "import zipfile\n",
    "import os\n",
    "from io import BytesIO\n",
    "import tiktoken\n",
    "\n",
    "tiktoken_encoder = tiktoken.encoding_for_model('gpt-3.5-turbo')\n",
    "# 准备一个列表来存储结果\n",
    "result_list = []\n",
    "\n",
    "# 临时内存文件\n",
    "new_zip_data = BytesIO()\n",
    "\n",
    "# 打开原始 ZIP 文件\n",
    "with zipfile.ZipFile(r'D:\\jetbrains projects\\pycharm\\GPT-Java-Tester\\dataset\\lang_1_fixed.zip', 'r') as zip_ref:\n",
    "    # 创建一个新的 ZipFile 对象，用于存储修改后的内容\n",
    "    with zipfile.ZipFile(new_zip_data, 'w', zipfile.ZIP_DEFLATED) as new_zip_ref:\n",
    "        # 遍历原始 ZIP 文件中的所有文件\n",
    "        for file_info in zip_ref.infolist():\n",
    "            filepath = file_info.filename\n",
    "            file_data = zip_ref.read(filepath)\n",
    "            # 检查是否是需要修改的文件\n",
    "            if filepath.endswith(\".java\") and \"src/main/java\" in filepath:\n",
    "                # 对文件内容进行解码\n",
    "                decoded_data = file_data.decode(errors=\"replace\")\n",
    "                # 处理文件内容\n",
    "                clazz_name, new_file_data, method_infos = get_class_info(decoded_data)\n",
    "                # 如果处理后的文件不为空，则使用修改后的文件数据\n",
    "                if clazz_name is not None:\n",
    "                    file_data = new_file_data\n",
    "                    package_reference = filepath.split(\"src/main/java/\")[1][:-5].replace(\"/\", \".\").replace(\n",
    "                        f\".{clazz_name}\", \"\")\n",
    "                    # 假设 tiktoken_encoder 是您定义的一个对象，用于编码文件数据\n",
    "                    tokens = len(tiktoken_encoder.encode(file_data))\n",
    "                    if tokens > 15000:\n",
    "                        print(clazz_name)\n",
    "                    result_list.append({\n",
    "                        \"class_name\": clazz_name,\n",
    "                        \"content\": file_data,  # 再次解码以存储为字符串\n",
    "                        \"methods\": sorted(method_infos, key=lambda x: x[\"start\"]),\n",
    "                        \"tokens\": tokens,\n",
    "                        \"package_reference\": package_reference,\n",
    "                        \"lines\": len(file_data.splitlines())\n",
    "                    })\n",
    "            # 将原始或修改后的文件数据写入新的 ZIP 文件中\n",
    "            new_zip_ref.writestr(file_info, file_data)\n",
    "\n",
    "# 关闭内存中的 ZIP 文件\n",
    "new_zip_data.seek(0)\n",
    "\n",
    "# 删除原始 ZIP 文件\n",
    "os.remove(r'D:\\jetbrains projects\\pycharm\\GPT-Java-Tester\\dataset\\lang_1_fixed.zip')\n",
    "\n",
    "# 将新的 ZIP 文件数据写入到原始文件路径\n",
    "with open(r'D:\\jetbrains projects\\pycharm\\GPT-Java-Tester\\dataset\\lang_1_fixed.zip', 'wb') as f:\n",
    "    f.write(new_zip_data.read())\n",
    "\n",
    "# 关闭内存文件\n",
    "new_zip_data.close()\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-24T09:56:26.262726600Z",
     "start_time": "2024-01-24T09:56:23.268583500Z"
    }
   },
   "execution_count": 88
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clazz num: 87\n",
      "method num: 1728\n",
      "\n",
      "package org.apache.commons.lang3;\n",
      "\n",
      "import java.lang.reflect.Array;\n",
      "import java.util.Arrays;\n",
      "import java.util.BitSet;\n",
      "import java.util.HashMap;\n",
      "import java.util.Map;\n",
      "\n",
      "import org.apache.commons.lang3.builder.EqualsBuilder;\n",
      "import org.apache.commons.lang3.builder.HashCodeBuilder;\n",
      "import org.apache.commons.lang3.builder.ToStringBuilder;\n",
      "import org.apache.commons.lang3.builder.ToStringStyle;\n",
      "import org.apache.commons.lang3.mutable.MutableInt;\n",
      "\n",
      "\n",
      "public class ArrayUtils {\n",
      "\n",
      "    \n",
      "    public static final Object[] EMPTY_OBJECT_ARRAY = new Object[0];\n",
      "    \n",
      "    public static final Class<?>[] EMPTY_CLASS_ARRAY = new Class[0];\n",
      "    \n",
      "    public static final String[] EMPTY_STRING_ARRAY = new String[0];\n",
      "    \n",
      "    public static final long[] EMPTY_LONG_ARRAY = new long[0];\n",
      "    \n",
      "    public static final Long[] EMPTY_LONG_OBJECT_ARRAY = new Long[0];\n",
      "    \n",
      "    public static final int[] EMPTY_INT_ARRAY = new int[0];\n",
      "    \n",
      "    public static final Integer[] EMPTY_INTEGER_OBJECT_ARRAY = new Integer[0];\n",
      "    \n",
      "    public static final short[] EMPTY_SHORT_ARRAY = new short[0];\n",
      "    \n",
      "    public static final Short[] EMPTY_SHORT_OBJECT_ARRAY = new Short[0];\n",
      "    \n",
      "    public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];\n",
      "    \n",
      "    public static final Byte[] EMPTY_BYTE_OBJECT_ARRAY = new Byte[0];\n",
      "    \n",
      "    public static final double[] EMPTY_DOUBLE_ARRAY = new double[0];\n",
      "    \n",
      "    public static final Double[] EMPTY_DOUBLE_OBJECT_ARRAY = new Double[0];\n",
      "    \n",
      "    public static final float[] EMPTY_FLOAT_ARRAY = new float[0];\n",
      "    \n",
      "    public static final Float[] EMPTY_FLOAT_OBJECT_ARRAY = new Float[0];\n",
      "    \n",
      "    public static final boolean[] EMPTY_BOOLEAN_ARRAY = new boolean[0];\n",
      "    \n",
      "    public static final Boolean[] EMPTY_BOOLEAN_OBJECT_ARRAY = new Boolean[0];\n",
      "    \n",
      "    public static final char[] EMPTY_CHAR_ARRAY = new char[0];\n",
      "    \n",
      "    public static final Character[] EMPTY_CHARACTER_OBJECT_ARRAY = new Character[0];\n",
      "\n",
      "    \n",
      "    public static final int INDEX_NOT_FOUND = -1;\n",
      "\n",
      "    \n",
      "    public ArrayUtils() {\n",
      "      super();\n",
      "    }\n",
      "\n",
      "\n",
      "    // NOTE: Cannot use {@code} to enclose text which includes {}, but <code></code> is OK\n",
      "\n",
      "\n",
      "    // Basic methods handling multi-dimensional arrays\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "    public static String toString(final Object array) {\n",
      "        return toString(array, \"{}\");\n",
      "    }\n",
      "\n",
      "    \n",
      "toString(Object, String) : String\n",
      "    \n",
      "hashCode(Object) : int\n",
      "    \n",
      "isEquals(Object, Object) : boolean\n",
      "    // To map\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "toMap(Object[]) : Map<Object, Object>\n",
      "    // Generic array\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "toArray(T...) : T[]\n",
      "    // Clone\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "clone(T[]) : T[]\n",
      "    \n",
      "clone(long[]) : long[]\n",
      "    \n",
      "clone(int[]) : int[]\n",
      "    \n",
      "clone(short[]) : short[]\n",
      "    \n",
      "clone(char[]) : char[]\n",
      "    \n",
      "clone(byte[]) : byte[]\n",
      "    \n",
      "clone(double[]) : double[]\n",
      "    \n",
      "clone(float[]) : float[]\n",
      "    \n",
      "clone(boolean[]) : boolean[]\n",
      "    // nullToEmpty\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "nullToEmpty(Object[]) : Object[]\n",
      "    \n",
      "nullToEmpty(String[]) : String[]\n",
      "    \n",
      "nullToEmpty(long[]) : long[]\n",
      "    \n",
      "nullToEmpty(int[]) : int[]\n",
      "    \n",
      "nullToEmpty(short[]) : short[]\n",
      "    \n",
      "nullToEmpty(char[]) : char[]\n",
      "    \n",
      "nullToEmpty(byte[]) : byte[]\n",
      "    \n",
      "nullToEmpty(double[]) : double[]\n",
      "    \n",
      "nullToEmpty(float[]) : float[]\n",
      "    \n",
      "nullToEmpty(boolean[]) : boolean[]\n",
      "    \n",
      "nullToEmpty(Long[]) : Long[]\n",
      "    \n",
      "nullToEmpty(Integer[]) : Integer[]\n",
      "    \n",
      "nullToEmpty(Short[]) : Short[]\n",
      "    \n",
      "nullToEmpty(Character[]) : Character[]\n",
      "    \n",
      "nullToEmpty(Byte[]) : Byte[]\n",
      "    \n",
      "nullToEmpty(Double[]) : Double[]\n",
      "    \n",
      "nullToEmpty(Float[]) : Float[]\n",
      "    \n",
      "nullToEmpty(Boolean[]) : Boolean[]\n",
      "    // Subarrays\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "subarray(T[], int, int) : T[]\n",
      "    \n",
      "subarray(long[], int, int) : long[]\n",
      "    \n",
      "subarray(int[], int, int) : int[]\n",
      "    \n",
      "subarray(short[], int, int) : short[]\n",
      "    \n",
      "subarray(char[], int, int) : char[]\n",
      "    \n",
      "subarray(byte[], int, int) : byte[]\n",
      "    \n",
      "subarray(double[], int, int) : double[]\n",
      "    \n",
      "subarray(float[], int, int) : float[]\n",
      "    \n",
      "subarray(boolean[], int, int) : boolean[]\n",
      "    // Is same length\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "isSameLength(Object[], Object[]) : boolean\n",
      "    \n",
      "isSameLength(long[], long[]) : boolean\n",
      "    \n",
      "isSameLength(int[], int[]) : boolean\n",
      "    \n",
      "isSameLength(short[], short[]) : boolean\n",
      "    \n",
      "isSameLength(char[], char[]) : boolean\n",
      "    \n",
      "isSameLength(byte[], byte[]) : boolean\n",
      "    \n",
      "isSameLength(double[], double[]) : boolean\n",
      "    \n",
      "isSameLength(float[], float[]) : boolean\n",
      "    \n",
      "isSameLength(boolean[], boolean[]) : boolean\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "getLength(Object) : int\n",
      "    \n",
      "isSameType(Object, Object) : boolean\n",
      "    // Reverse\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "reverse(Object[]) : void\n",
      "    \n",
      "reverse(long[]) : void\n",
      "    \n",
      "reverse(int[]) : void\n",
      "    \n",
      "reverse(short[]) : void\n",
      "    \n",
      "reverse(char[]) : void\n",
      "    \n",
      "reverse(byte[]) : void\n",
      "    \n",
      "reverse(double[]) : void\n",
      "    \n",
      "reverse(float[]) : void\n",
      "    \n",
      "reverse(boolean[]) : void\n",
      "    // IndexOf search\n",
      "    // ----------------------------------------------------------------------\n",
      "\n",
      "    // Object IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(Object[], Object) : int\n",
      "    \n",
      "indexOf(Object[], Object, int) : int\n",
      "    \n",
      "lastIndexOf(Object[], Object) : int\n",
      "    \n",
      "lastIndexOf(Object[], Object, int) : int\n",
      "    \n",
      "contains(Object[], Object) : boolean\n",
      "    // long IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(long[], long) : int\n",
      "    \n",
      "indexOf(long[], long, int) : int\n",
      "    \n",
      "lastIndexOf(long[], long) : int\n",
      "    \n",
      "lastIndexOf(long[], long, int) : int\n",
      "    \n",
      "contains(long[], long) : boolean\n",
      "    // int IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(int[], int) : int\n",
      "    \n",
      "indexOf(int[], int, int) : int\n",
      "    \n",
      "lastIndexOf(int[], int) : int\n",
      "    \n",
      "lastIndexOf(int[], int, int) : int\n",
      "    \n",
      "contains(int[], int) : boolean\n",
      "    // short IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(short[], short) : int\n",
      "    \n",
      "indexOf(short[], short, int) : int\n",
      "    \n",
      "lastIndexOf(short[], short) : int\n",
      "    \n",
      "lastIndexOf(short[], short, int) : int\n",
      "    \n",
      "contains(short[], short) : boolean\n",
      "    // char IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(char[], char) : int\n",
      "    \n",
      "indexOf(char[], char, int) : int\n",
      "    \n",
      "lastIndexOf(char[], char) : int\n",
      "    \n",
      "lastIndexOf(char[], char, int) : int\n",
      "    \n",
      "contains(char[], char) : boolean\n",
      "    // byte IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(byte[], byte) : int\n",
      "    \n",
      "indexOf(byte[], byte, int) : int\n",
      "    \n",
      "lastIndexOf(byte[], byte) : int\n",
      "    \n",
      "lastIndexOf(byte[], byte, int) : int\n",
      "    \n",
      "contains(byte[], byte) : boolean\n",
      "    // double IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(double[], double) : int\n",
      "    \n",
      "indexOf(double[], double, double) : int\n",
      "    \n",
      "indexOf(double[], double, int) : int\n",
      "    \n",
      "indexOf(double[], double, int, double) : int\n",
      "    \n",
      "lastIndexOf(double[], double) : int\n",
      "    \n",
      "lastIndexOf(double[], double, double) : int\n",
      "    \n",
      "lastIndexOf(double[], double, int) : int\n",
      "    \n",
      "lastIndexOf(double[], double, int, double) : int\n",
      "    \n",
      "contains(double[], double) : boolean\n",
      "    \n",
      "contains(double[], double, double) : boolean\n",
      "    // float IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(float[], float) : int\n",
      "    \n",
      "indexOf(float[], float, int) : int\n",
      "    \n",
      "lastIndexOf(float[], float) : int\n",
      "    \n",
      "lastIndexOf(float[], float, int) : int\n",
      "    \n",
      "contains(float[], float) : boolean\n",
      "    // boolean IndexOf\n",
      "    //-----------------------------------------------------------------------\n",
      "    \n",
      "indexOf(boolean[], boolean) : int\n",
      "    \n",
      "indexOf(boolean[], boolean, int) : int\n",
      "    \n",
      "lastIndexOf(boolean[], boolean) : int\n",
      "    \n",
      "lastIndexOf(boolean[], boolean, int) : int\n",
      "    \n",
      "contains(boolean[], boolean) : boolean\n",
      "    // Primitive/Object array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "\n",
      "    // Character array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Character[]) : char[]\n",
      "    \n",
      "toPrimitive(Character[], char) : char[]\n",
      "    \n",
      "toObject(char[]) : Character[]\n",
      "    // Long array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Long[]) : long[]\n",
      "    \n",
      "toPrimitive(Long[], long) : long[]\n",
      "    \n",
      "toObject(long[]) : Long[]\n",
      "    // Int array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Integer[]) : int[]\n",
      "    \n",
      "toPrimitive(Integer[], int) : int[]\n",
      "    \n",
      "toObject(int[]) : Integer[]\n",
      "    // Short array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Short[]) : short[]\n",
      "    \n",
      "toPrimitive(Short[], short) : short[]\n",
      "    \n",
      "toObject(short[]) : Short[]\n",
      "    // Byte array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Byte[]) : byte[]\n",
      "    \n",
      "toPrimitive(Byte[], byte) : byte[]\n",
      "    \n",
      "toObject(byte[]) : Byte[]\n",
      "    // Double array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Double[]) : double[]\n",
      "    \n",
      "toPrimitive(Double[], double) : double[]\n",
      "    \n",
      "toObject(double[]) : Double[]\n",
      "    //   Float array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Float[]) : float[]\n",
      "    \n",
      "toPrimitive(Float[], float) : float[]\n",
      "    \n",
      "toObject(float[]) : Float[]\n",
      "    // Boolean array converters\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "toPrimitive(Boolean[]) : boolean[]\n",
      "    \n",
      "toPrimitive(Boolean[], boolean) : boolean[]\n",
      "    \n",
      "toObject(boolean[]) : Boolean[]\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "isEmpty(Object[]) : boolean\n",
      "    \n",
      "isEmpty(long[]) : boolean\n",
      "    \n",
      "isEmpty(int[]) : boolean\n",
      "    \n",
      "isEmpty(short[]) : boolean\n",
      "    \n",
      "isEmpty(char[]) : boolean\n",
      "    \n",
      "isEmpty(byte[]) : boolean\n",
      "    \n",
      "isEmpty(double[]) : boolean\n",
      "    \n",
      "isEmpty(float[]) : boolean\n",
      "    \n",
      "isEmpty(boolean[]) : boolean\n",
      "    // ----------------------------------------------------------------------\n",
      "    \n",
      "isNotEmpty(T[]) : boolean\n",
      "    \n",
      "isNotEmpty(long[]) : boolean\n",
      "    \n",
      "isNotEmpty(int[]) : boolean\n",
      "    \n",
      "isNotEmpty(short[]) : boolean\n",
      "    \n",
      "isNotEmpty(char[]) : boolean\n",
      "    \n",
      "isNotEmpty(byte[]) : boolean\n",
      "    \n",
      "isNotEmpty(double[]) : boolean\n",
      "    \n",
      "isNotEmpty(float[]) : boolean\n",
      "    \n",
      "isNotEmpty(boolean[]) : boolean\n",
      "    \n",
      "addAll(T[], T...) : T[]\n",
      "    \n",
      "addAll(boolean[], boolean...) : boolean[]\n",
      "    \n",
      "addAll(char[], char...) : char[]\n",
      "    \n",
      "addAll(byte[], byte...) : byte[]\n",
      "    \n",
      "addAll(short[], short...) : short[]\n",
      "    \n",
      "addAll(int[], int...) : int[]\n",
      "    \n",
      "addAll(long[], long...) : long[]\n",
      "    \n",
      "addAll(float[], float...) : float[]\n",
      "    \n",
      "addAll(double[], double...) : double[]\n",
      "    \n",
      "add(T[], T) : T[]\n",
      "    \n",
      "add(boolean[], boolean) : boolean[]\n",
      "    \n",
      "add(byte[], byte) : byte[]\n",
      "    \n",
      "add(char[], char) : char[]\n",
      "    \n",
      "add(double[], double) : double[]\n",
      "    \n",
      "add(float[], float) : float[]\n",
      "    \n",
      "add(int[], int) : int[]\n",
      "    \n",
      "add(long[], long) : long[]\n",
      "    \n",
      "add(short[], short) : short[]\n",
      "    \n",
      "copyArrayGrow1(Object, Class<>) : Object\n",
      "    \n",
      "add(T[], int, T) : T[]\n",
      "    \n",
      "add(boolean[], int, boolean) : boolean[]\n",
      "    \n",
      "add(char[], int, char) : char[]\n",
      "    \n",
      "add(byte[], int, byte) : byte[]\n",
      "    \n",
      "add(short[], int, short) : short[]\n",
      "    \n",
      "add(int[], int, int) : int[]\n",
      "    \n",
      "add(long[], int, long) : long[]\n",
      "    \n",
      "add(float[], int, float) : float[]\n",
      "    \n",
      "add(double[], int, double) : double[]\n",
      "    \n",
      "add(Object, int, Object, Class<>) : Object\n",
      "    \n",
      "    @SuppressWarnings(\"unchecked\") // remove() always creates an array of the same type as its input\n",
      "remove(T[], int) : T[]\n",
      "    \n",
      "removeElement(T[], Object) : T[]\n",
      "    \n",
      "remove(boolean[], int) : boolean[]\n",
      "    \n",
      "removeElement(boolean[], boolean) : boolean[]\n",
      "    \n",
      "remove(byte[], int) : byte[]\n",
      "    \n",
      "removeElement(byte[], byte) : byte[]\n",
      "    \n",
      "remove(char[], int) : char[]\n",
      "    \n",
      "removeElement(char[], char) : char[]\n",
      "    \n",
      "remove(double[], int) : double[]\n",
      "    \n",
      "removeElement(double[], double) : double[]\n",
      "    \n",
      "remove(float[], int) : float[]\n",
      "    \n",
      "removeElement(float[], float) : float[]\n",
      "    \n",
      "remove(int[], int) : int[]\n",
      "    \n",
      "removeElement(int[], int) : int[]\n",
      "    \n",
      "remove(long[], int) : long[]\n",
      "    \n",
      "removeElement(long[], long) : long[]\n",
      "    \n",
      "remove(short[], int) : short[]\n",
      "    \n",
      "removeElement(short[], short) : short[]\n",
      "    \n",
      "remove(Object, int) : Object\n",
      "    \n",
      "    @SuppressWarnings(\"unchecked\") // removeAll() always creates an array of the same type as its input\n",
      "removeAll(T[], int...) : T[]\n",
      "    \n",
      "removeElements(T[], T...) : T[]\n",
      "    \n",
      "removeAll(byte[], int...) : byte[]\n",
      "    \n",
      "removeElements(byte[], byte...) : byte[]\n",
      "    \n",
      "removeAll(short[], int...) : short[]\n",
      "    \n",
      "removeElements(short[], short...) : short[]\n",
      "    \n",
      "removeAll(int[], int...) : int[]\n",
      "    \n",
      "removeElements(int[], int...) : int[]\n",
      "    \n",
      "removeAll(char[], int...) : char[]\n",
      "    \n",
      "removeElements(char[], char...) : char[]\n",
      "    \n",
      "removeAll(long[], int...) : long[]\n",
      "    \n",
      "removeElements(long[], long...) : long[]\n",
      "    \n",
      "removeAll(float[], int...) : float[]\n",
      "    \n",
      "removeElements(float[], float...) : float[]\n",
      "    \n",
      "removeAll(double[], int...) : double[]\n",
      "    \n",
      "removeElements(double[], double...) : double[]\n",
      "    \n",
      "removeAll(boolean[], int...) : boolean[]\n",
      "    \n",
      "removeElements(boolean[], boolean...) : boolean[]\n",
      "    \n",
      "    // package protected for access by unit tests\n",
      "removeAll(Object, int...) : Object\n",
      "    \n",
      "    // package protected for access by unit tests\n",
      "removeAll(Object, BitSet) : Object\n"
     ]
    }
   ],
   "source": [
    "print(f\"clazz num: {len(result_list)}\")\n",
    "print(f\"method num: {sum([len(clazz['methods']) for clazz in result_list])}\")\n",
    "print(result_list[1][\"methods\"][0][\"compressed_content\"])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-24T09:56:27.152436Z",
     "start_time": "2024-01-24T09:56:27.144003600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# 验证数据集正确性\n",
    "def has_overlap(intervals):\n",
    "    intervals.sort(key=lambda x: x[0])  # 按照起点进行排序\n",
    "    for i in range(1, len(intervals)):\n",
    "        if intervals[i][0] < intervals[i - 1][1]:  # 如果下一个区间的起点小于上一个区间的终点，说明有重合\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "\n",
    "li = []\n",
    "for class_info in result_list:\n",
    "    for method_info in class_info[\"methods\"]:\n",
    "        li.append((class_info[\"class_name\"], method_info[\"name\"], method_info[\"start\"], method_info[\"end\"]))\n",
    "    if has_overlap([(method_info[\"start\"], method_info[\"end\"]) for method_info in class_info[\"methods\"]]):\n",
    "        print(class_info[\"class_name\"])\n",
    "        assert False\n",
    "        break\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(li)\n",
    "df.to_csv(\"method_info.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-24T09:58:20.733028600Z",
     "start_time": "2024-01-24T09:58:20.720132800Z"
    }
   },
   "execution_count": 90
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "outputs": [],
   "source": [
    "import pickle\n",
    "import gzip\n",
    "\n",
    "# 保存压缩后的对象到文件\n",
    "with open(\"lang_1_fixed.pkl\", \"wb\") as f:\n",
    "    pickle.dump({\"data\": sorted(result_list, key=lambda x: x[\"class_name\"])}, f)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-24T09:58:22.806285400Z",
     "start_time": "2024-01-24T09:58:22.780723900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "with open(\"lang_1_fixed_info.pkl\", \"rb\") as f:\n",
    "    dataset_info = pickle.load(f)\n",
    "\n",
    "with open(\"lang_1_fixed.pkl\", \"rb\") as f:\n",
    "    dataset = pickle.load(f)[\"data\"]\n",
    "\n",
    "\n",
    "# 假设你已经有了 dataset_info 和 dataset\n",
    "\n",
    "def sort_dataset(dataset, dataset_info):\n",
    "    # 创建一个字典，用于快速查找class_name和method_name的顺序\n",
    "    class_order = {info['class_name']: idx for idx, info in enumerate(dataset_info)}\n",
    "    method_order = {info['method_name']: idx for idx, info in enumerate(dataset_info)}\n",
    "\n",
    "    # 定义一个辅助函数，用于获取class_name的排序键\n",
    "    def get_class_order_key(item):\n",
    "        # 如果class_name不在dataset_info中，返回一个大数使其排在最后\n",
    "        return class_order.get(item['class_name'], float('inf'))\n",
    "\n",
    "    # 定义一个辅助函数，用于获取method_name的排序键\n",
    "    def get_method_order_key(item):\n",
    "        # 如果method_name不在dataset_info中，返回一个大数使其排在最后\n",
    "        return method_order.get(item['name'], float('inf'))\n",
    "\n",
    "    # 首先按照class_name对dataset进行排序\n",
    "    sorted_dataset = sorted(dataset, key=get_class_order_key)\n",
    "\n",
    "    # 然后对每个元素的methods按照method_name进行排序\n",
    "    for item in sorted_dataset:\n",
    "        item['methods'] = sorted(item['methods'], key=get_method_order_key)\n",
    "\n",
    "    return sorted_dataset\n",
    "\n",
    "\n",
    "# 对数据集进行排序\n",
    "sorted_dataset = sort_dataset(dataset, dataset_info)\n",
    "\n",
    "# 打印排序后的数据集\n",
    "for item in sorted_dataset:\n",
    "    print(item['class_name'])\n",
    "    for method in item['methods']:\n",
    "        print('  ', method['name'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# 写入\n",
    "with open(\"lang_1_fixed.pkl\", \"wb\") as f:\n",
    "    pickle.dump({\"data\": sorted_dataset}, f)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-23T03:50:52.876834900Z",
     "start_time": "2024-01-23T03:50:52.853521300Z"
    }
   },
   "execution_count": 44
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "\n",
    "def remove_comments(input_string: str) -> str:\n",
    "    pattern = r'/\\*\\*?\\s.*?\\*/'\n",
    "    return re.sub(pattern, '', input_string, flags=re.DOTALL)\n",
    "\n",
    "\n",
    "def process_java_file(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8', errors='replace') as file:\n",
    "        content = file.read()\n",
    "        content = remove_comments(content)\n",
    "    with open(file_path, 'w', encoding='utf-8', errors='replace') as file:\n",
    "        file.write(content)\n",
    "\n",
    "\n",
    "def process_directory(directory_path):\n",
    "    for root, dirs, files in os.walk(directory_path):\n",
    "        for file in files:\n",
    "            if file.endswith(\".java\"):\n",
    "                file_path = os.path.join(root, file)\n",
    "                process_java_file(file_path)\n",
    "\n",
    "\n",
    "directory_path = r\"D:\\jetbrains projects\\pycharm\\GPT-Java-Tester\\dataset\\src\\main\\java\\org\\apache\\commons\\lang3\"\n",
    "process_directory(directory_path)\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-23T08:41:48.519434Z",
     "start_time": "2024-01-23T08:41:48.418747200Z"
    }
   },
   "execution_count": 46
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [],
   "source": [
    "from pydantic import BaseModel\n",
    "from typing import *\n",
    "\n",
    "\n",
    "class Dataset(BaseModel):\n",
    "    class ClassInfo(BaseModel):\n",
    "        class MethodInfo(BaseModel):\n",
    "            start: int\n",
    "            end: int\n",
    "            content: str\n",
    "\n",
    "        content: str\n",
    "        tokens: int\n",
    "        lines: int\n",
    "        methods: dict[str, MethodInfo]\n",
    "\n",
    "        def __iter__(self) -> Iterator[Tuple[str, MethodInfo]]:\n",
    "            return self\n",
    "\n",
    "        def __next__(self) -> Tuple[str, MethodInfo]:\n",
    "            for method_name, method_info in self.methods.items():\n",
    "                return method_name, method_info\n",
    "\n",
    "    data: dict[str, ClassInfo]\n",
    "\n",
    "    def __iter__(self) -> Iterator[Tuple[str, ClassInfo]]:\n",
    "        return self\n",
    "\n",
    "    def __next__(self) -> Tuple[str, ClassInfo]:\n",
    "        for classname, classinfo in self.data.items():\n",
    "            return classname, classinfo\n",
    "\n",
    "    def get_item(self, class_name, method_name):\n",
    "        class_info = self.data[class_name]\n",
    "        method_info = class_info.methods[method_name]\n",
    "        return class_info, method_info\n",
    "\n",
    "\n",
    "d = Dataset(**obj)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-29T12:44:07.661537200Z",
     "start_time": "2023-12-29T12:44:07.651040400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('EnumUtils', ClassInfo(content='/*\\n * Licensed to the Apache Software Foundation (ASF) under one or more\\n * contributor license agreements.  See the NOTICE file distributed with\\n * this work for additional information regarding copyright ownership.\\n * The ASF licenses this file to You under the Apache License, Version 2.0\\n * (the \"License\"); you may not use this file except in compliance with\\n * the License.  You may obtain a copy of the License at\\n *\\n *      http://www.apache.org/licenses/LICENSE-2.0\\n *\\n * Unless required by applicable law or agreed to in writing, software\\n * distributed under the License is distributed on an \"AS IS\" BASIS,\\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n * See the License for the specific language governing permissions and\\n * limitations under the License.\\n */\\npackage org.apache.commons.lang3;\\n\\nimport java.util.ArrayList;\\nimport java.util.Arrays;\\nimport java.util.Collections;\\nimport java.util.EnumSet;\\nimport java.util.LinkedHashMap;\\nimport java.util.List;\\nimport java.util.Map;\\n\\n/**\\n * <p>Utility library to provide helper methods for Java enums.</p>\\n *\\n * <p>#ThreadSafe#</p>\\n *\\n * @since 3.0\\n * @version $Id$\\n */\\npublic class EnumUtils {\\n\\n    private static final String NULL_ELEMENTS_NOT_PERMITTED = \"null elements not permitted\";\\n    private static final String CANNOT_STORE_S_S_VALUES_IN_S_BITS = \"Cannot store %s %s values in %s bits\";\\n    private static final String S_DOES_NOT_SEEM_TO_BE_AN_ENUM_TYPE = \"%s does not seem to be an Enum type\";\\n    private static final String ENUM_CLASS_MUST_BE_DEFINED = \"EnumClass must be defined.\";\\n\\n    /**\\n     * This constructor is public to permit tools that require a JavaBean\\n     * instance to operate.\\n     */\\n    public EnumUtils() {\\n    }\\n\\n    /**\\n     * <p>Gets the {@code Map} of enums by name.</p>\\n     *\\n     * <p>This method is useful when you need a map of enums by name.</p>\\n     *\\n     * @param <E> the type of the enumeration\\n     * @param enumClass  the class of the enum to query, not null\\n     * @return the modifiable map of enum names to enums, never null\\n     */\\n    public static <E extends Enum<E>> Map<String, E> getEnumMap(final Class<E> enumClass) {\\n        final Map<String, E> map = new LinkedHashMap<String, E>();\\n        for (final E e: enumClass.getEnumConstants()) {\\n            map.put(e.name(), e);\\n        }\\n        return map;\\n    }\\n\\n    /**\\n     * <p>Gets the {@code List} of enums.</p>\\n     *\\n     * <p>This method is useful when you need a list of enums rather than an array.</p>\\n     *\\n     * @param <E> the type of the enumeration\\n     * @param enumClass  the class of the enum to query, not null\\n     * @return the modifiable list of enums, never null\\n     */\\n    public static <E extends Enum<E>> List<E> getEnumList(final Class<E> enumClass) {\\n        return new ArrayList<E>(Arrays.asList(enumClass.getEnumConstants()));\\n    }\\n\\n    /**\\n     * <p>Checks if the specified name is a valid enum for the class.</p>\\n     *\\n     * <p>This method differs from {@link Enum#valueOf} in that checks if the name is\\n     * a valid enum without needing to catch the exception.</p>\\n     *\\n     * @param <E> the type of the enumeration\\n     * @param enumClass  the class of the enum to query, not null\\n     * @param enumName   the enum name, null returns false\\n     * @return true if the enum name is valid, otherwise false\\n     */\\n    public static <E extends Enum<E>> boolean isValidEnum(final Class<E> enumClass, final String enumName) {\\n        if (enumName == null) {\\n            return false;\\n        }\\n        try {\\n            Enum.valueOf(enumClass, enumName);\\n            return true;\\n        } catch (final IllegalArgumentException ex) {\\n            return false;\\n        }\\n    }\\n\\n    /**\\n     * <p>Gets the enum for the class, returning {@code null} if not found.</p>\\n     *\\n     * <p>This method differs from {@link Enum#valueOf} in that it does not throw an exception\\n     * for an invalid enum name.</p>\\n     *\\n     * @param <E> the type of the enumeration\\n     * @param enumClass  the class of the enum to query, not null\\n     * @param enumName   the enum name, null returns null\\n     * @return the enum, null if not found\\n     */\\n    public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {\\n        if (enumName == null) {\\n            return null;\\n        }\\n        try {\\n            return Enum.valueOf(enumClass, enumName);\\n        } catch (final IllegalArgumentException ex) {\\n            return null;\\n        }\\n    }\\n\\n    /**\\n     * <p>Creates a long bit vector representation of the given subset of an Enum.</p>\\n     *\\n     * <p>This generates a value that is usable by {@link EnumUtils#processBitVector}.</p>\\n     *\\n     * <p>Do not use this method if you have more than 64 values in your Enum, as this\\n     * would create a value greater than a long can hold.</p>\\n     *\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param values    the values we want to convert, not {@code null}, neither containing {@code null}\\n     * @param <E>       the type of the enumeration\\n     * @return a long whose value provides a binary representation of the given set of enum values.\\n     * @throws NullPointerException if {@code enumClass} or {@code values} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class or has more than 64 values,\\n     *                                  or if any {@code values} {@code null}\\n     * @since 3.0.1\\n     * @see #generateBitVectors(Class, Iterable)\\n     */\\n    public static <E extends Enum<E>> long generateBitVector(final Class<E> enumClass, final Iterable<E> values) {\\n        checkBitVectorable(enumClass);\\n        Validate.notNull(values);\\n        long total = 0;\\n        for (final E constant : values) {\\n            Validate.isTrue(constant != null, NULL_ELEMENTS_NOT_PERMITTED);\\n            total |= 1 << constant.ordinal();\\n        }\\n        return total;\\n    }\\n\\n    /**\\n     * <p>Creates a bit vector representation of the given subset of an Enum using as many {@code long}s as needed.</p>\\n     *\\n     * <p>This generates a value that is usable by {@link EnumUtils#processBitVectors}.</p>\\n     *\\n     * <p>Use this method if you have more than 64 values in your Enum.</p>\\n     *\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param values    the values we want to convert, not {@code null}, neither containing {@code null}\\n     * @param <E>       the type of the enumeration\\n     * @return a long[] whose values provide a binary representation of the given set of enum values\\n     *         with least significant digits rightmost.\\n     * @throws NullPointerException if {@code enumClass} or {@code values} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class, or if any {@code values} {@code null}\\n     * @since 3.2\\n     */\\n    public static <E extends Enum<E>> long[] generateBitVectors(final Class<E> enumClass, final Iterable<E> values) {\\n        asEnum(enumClass);\\n        Validate.notNull(values);\\n        final EnumSet<E> condensed = EnumSet.noneOf(enumClass);\\n        for (final E constant : values) {\\n            Validate.isTrue(constant != null, NULL_ELEMENTS_NOT_PERMITTED);\\n            condensed.add(constant);\\n        }\\n        final long[] result = new long[(enumClass.getEnumConstants().length - 1) / Long.SIZE + 1];\\n        for (final E value : condensed) {\\n            result[value.ordinal() / Long.SIZE] |= 1 << (value.ordinal() % Long.SIZE);\\n        }\\n        ArrayUtils.reverse(result);\\n        return result;\\n    }\\n\\n    /**\\n     * <p>Creates a long bit vector representation of the given array of Enum values.</p>\\n     *\\n     * <p>This generates a value that is usable by {@link EnumUtils#processBitVector}.</p>\\n     *\\n     * <p>Do not use this method if you have more than 64 values in your Enum, as this\\n     * would create a value greater than a long can hold.</p>\\n     *\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param values    the values we want to convert, not {@code null}\\n     * @param <E>       the type of the enumeration\\n     * @return a long whose value provides a binary representation of the given set of enum values.\\n     * @throws NullPointerException if {@code enumClass} or {@code values} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class or has more than 64 values\\n     * @since 3.0.1\\n     * @see #generateBitVectors(Class, Iterable)\\n     */\\n    public static <E extends Enum<E>> long generateBitVector(final Class<E> enumClass, final E... values) {\\n        Validate.noNullElements(values);\\n        return generateBitVector(enumClass, Arrays.<E> asList(values));\\n    }\\n\\n    /**\\n     * <p>Creates a bit vector representation of the given subset of an Enum using as many {@code long}s as needed.</p>\\n     *\\n     * <p>This generates a value that is usable by {@link EnumUtils#processBitVectors}.</p>\\n     *\\n     * <p>Use this method if you have more than 64 values in your Enum.</p>\\n     *\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param values    the values we want to convert, not {@code null}, neither containing {@code null}\\n     * @param <E>       the type of the enumeration\\n     * @return a long[] whose values provide a binary representation of the given set of enum values\\n     *         with least significant digits rightmost.\\n     * @throws NullPointerException if {@code enumClass} or {@code values} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class, or if any {@code values} {@code null}\\n     * @since 3.2\\n     */\\n    public static <E extends Enum<E>> long[] generateBitVectors(final Class<E> enumClass, final E... values) {\\n        asEnum(enumClass);\\n        Validate.noNullElements(values);\\n        final EnumSet<E> condensed = EnumSet.noneOf(enumClass);\\n        Collections.addAll(condensed, values);\\n        final long[] result = new long[(enumClass.getEnumConstants().length - 1) / Long.SIZE + 1];\\n        for (final E value : condensed) {\\n            result[value.ordinal() / Long.SIZE] |= 1 << (value.ordinal() % Long.SIZE);\\n        }\\n        ArrayUtils.reverse(result);\\n        return result;\\n    }\\n\\n    /**\\n     * <p>Convert a long value created by {@link EnumUtils#generateBitVector} into the set of\\n     * enum values that it represents.</p>\\n     *\\n     * <p>If you store this value, beware any changes to the enum that would affect ordinal values.</p>\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param value     the long value representation of a set of enum values\\n     * @param <E>       the type of the enumeration\\n     * @return a set of enum values\\n     * @throws NullPointerException if {@code enumClass} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class or has more than 64 values\\n     * @since 3.0.1\\n     */\\n    public static <E extends Enum<E>> EnumSet<E> processBitVector(final Class<E> enumClass, final long value) {\\n        checkBitVectorable(enumClass).getEnumConstants();\\n        return processBitVectors(enumClass, value);\\n    }\\n\\n    /**\\n     * <p>Convert a {@code long[]} created by {@link EnumUtils#generateBitVectors} into the set of\\n     * enum values that it represents.</p>\\n     *\\n     * <p>If you store this value, beware any changes to the enum that would affect ordinal values.</p>\\n     * @param enumClass the class of the enum we are working with, not {@code null}\\n     * @param values     the long[] bearing the representation of a set of enum values, least significant digits rightmost, not {@code null}\\n     * @param <E>       the type of the enumeration\\n     * @return a set of enum values\\n     * @throws NullPointerException if {@code enumClass} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class\\n     * @since 3.2\\n     */\\n    public static <E extends Enum<E>> EnumSet<E> processBitVectors(final Class<E> enumClass, long... values) {\\n        final EnumSet<E> results = EnumSet.noneOf(asEnum(enumClass));\\n        values = ArrayUtils.clone(Validate.notNull(values));\\n        ArrayUtils.reverse(values);\\n        for (final E constant : enumClass.getEnumConstants()) {\\n            final int block = constant.ordinal() / Long.SIZE;\\n            if (block < values.length && (values[block] & 1 << (constant.ordinal() % Long.SIZE)) != 0) {\\n                results.add(constant);\\n            }\\n        }\\n        return results;\\n    }\\n\\n    /**\\n     * Validate that {@code enumClass} is compatible with representation in a {@code long}.\\n     * @param <E> the type of the enumeration\\n     * @param enumClass to check\\n     * @return {@code enumClass}\\n     * @throws NullPointerException if {@code enumClass} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class or has more than 64 values\\n     * @since 3.0.1\\n     */\\n    private static <E extends Enum<E>> Class<E> checkBitVectorable(final Class<E> enumClass) {\\n        final E[] constants = asEnum(enumClass).getEnumConstants();\\n        Validate.isTrue(constants.length <= Long.SIZE, CANNOT_STORE_S_S_VALUES_IN_S_BITS, constants.length,\\n            enumClass.getSimpleName(), Long.SIZE);\\n\\n        return enumClass;\\n    }\\n\\n    /**\\n     * Validate {@code enumClass}.\\n     * @param <E> the type of the enumeration\\n     * @param enumClass to check\\n     * @return {@code enumClass}\\n     * @throws NullPointerException if {@code enumClass} is {@code null}\\n     * @throws IllegalArgumentException if {@code enumClass} is not an enum class\\n     * @since 3.2\\n     */\\n    private static <E extends Enum<E>> Class<E> asEnum(final Class<E> enumClass) {\\n        Validate.notNull(enumClass, ENUM_CLASS_MUST_BE_DEFINED);\\n        Validate.isTrue(enumClass.isEnum(), S_DOES_NOT_SEEM_TO_BE_AN_ENUM_TYPE, enumClass);\\n        return enumClass;\\n    }\\n}\\n', tokens=3339, lines=311, methods={'getEnumMap': MethodInfo(start=58, end=64, content='public static <E extends Enum<E>> Map<String, E> getEnumMap(final Class<E> enumClass) {\\n    final Map<String, E> map = new LinkedHashMap<String, E>();\\n    for (final E e: enumClass.getEnumConstants()) {\\n        map.put(e.name(), e);\\n    }\\n    return map;\\n}'), 'getEnumList': MethodInfo(start=75, end=77, content='public static <E extends Enum<E>> List<E> getEnumList(final Class<E> enumClass) {\\n    return new ArrayList<E>(Arrays.asList(enumClass.getEnumConstants()));\\n}'), 'isValidEnum': MethodInfo(start=90, end=100, content='public static <E extends Enum<E>> boolean isValidEnum(final Class<E> enumClass, final String enumName) {\\n    if (enumName == null) {\\n        return false;\\n    }\\n    try {\\n        Enum.valueOf(enumClass, enumName);\\n        return true;\\n    } catch (final IllegalArgumentException ex) {\\n        return false;\\n    }\\n}'), 'getEnum': MethodInfo(start=113, end=122, content='public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {\\n    if (enumName == null) {\\n        return null;\\n    }\\n    try {\\n        return Enum.valueOf(enumClass, enumName);\\n    } catch (final IllegalArgumentException ex) {\\n        return null;\\n    }\\n}'), 'generateBitVector': MethodInfo(start=202, end=205, content='public static <E extends Enum<E>> long generateBitVector(final Class<E> enumClass, final E... values) {\\n    Validate.noNullElements(values);\\n    return generateBitVector(enumClass, Arrays.<E> asList(values));\\n}'), 'generateBitVectors': MethodInfo(start=223, end=234, content='public static <E extends Enum<E>> long[] generateBitVectors(final Class<E> enumClass, final E... values) {\\n    asEnum(enumClass);\\n    Validate.noNullElements(values);\\n    final EnumSet<E> condensed = EnumSet.noneOf(enumClass);\\n    Collections.addAll(condensed, values);\\n    final long[] result = new long[(enumClass.getEnumConstants().length - 1) / Long.SIZE + 1];\\n    for (final E value : condensed) {\\n        result[value.ordinal() / Long.SIZE] |= 1 << (value.ordinal() % Long.SIZE);\\n    }\\n    ArrayUtils.reverse(result);\\n    return result;\\n}'), 'processBitVector': MethodInfo(start=249, end=252, content='public static <E extends Enum<E>> EnumSet<E> processBitVector(final Class<E> enumClass, final long value) {\\n    checkBitVectorable(enumClass).getEnumConstants();\\n    return processBitVectors(enumClass, value);\\n}'), 'processBitVectors': MethodInfo(start=267, end=278, content='public static <E extends Enum<E>> EnumSet<E> processBitVectors(final Class<E> enumClass, long... values) {\\n    final EnumSet<E> results = EnumSet.noneOf(asEnum(enumClass));\\n    values = ArrayUtils.clone(Validate.notNull(values));\\n    ArrayUtils.reverse(values);\\n    for (final E constant : enumClass.getEnumConstants()) {\\n        final int block = constant.ordinal() / Long.SIZE;\\n        if (block < values.length && (values[block] & 1 << (constant.ordinal() % Long.SIZE)) != 0) {\\n            results.add(constant);\\n        }\\n    }\\n    return results;\\n}'), 'checkBitVectorable': MethodInfo(start=289, end=295, content='private static <E extends Enum<E>> Class<E> checkBitVectorable(final Class<E> enumClass) {\\n    final E[] constants = asEnum(enumClass).getEnumConstants();\\n    Validate.isTrue(constants.length <= Long.SIZE, CANNOT_STORE_S_S_VALUES_IN_S_BITS, constants.length,\\n        enumClass.getSimpleName(), Long.SIZE);\\n\\n    return enumClass;\\n}'), 'asEnum': MethodInfo(start=306, end=310, content='private static <E extends Enum<E>> Class<E> asEnum(final Class<E> enumClass) {\\n    Validate.notNull(enumClass, ENUM_CLASS_MUST_BE_DEFINED);\\n    Validate.isTrue(enumClass.isEnum(), S_DOES_NOT_SEEM_TO_BE_AN_ENUM_TYPE, enumClass);\\n    return enumClass;\\n}')}))\n"
     ]
    }
   ],
   "source": [
    "for k in d:\n",
    "    print(k)\n",
    "    break"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-29T12:44:33.311397800Z",
     "start_time": "2023-12-29T12:44:33.306400300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "outputs": [],
   "source": [
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "def etree_to_dict(t):\n",
    "    d = {t.tag: {} if t.attrib else None}\n",
    "    children = list(t)\n",
    "    if children:\n",
    "        dd = defaultdict(list)\n",
    "        for dc in map(etree_to_dict, children):\n",
    "            for k, v in dc.items():\n",
    "                dd[k].append(v)\n",
    "        d = {t.tag: {k: v for k, v in dd.items()}}\n",
    "    if t.attrib:\n",
    "        d[t.tag].update(('@' + k, v) for k, v in t.attrib.items())\n",
    "    if t.text:\n",
    "        text = t.text.strip()\n",
    "        if children or t.attrib:\n",
    "            if text:\n",
    "                d[t.tag]['#text'] = text\n",
    "        else:\n",
    "            d[t.tag] = text\n",
    "    return d\n",
    "\n",
    "\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "tree = ET.parse(r\"D:\\IDEA_Projects\\TestJavaCode\\target\\site\\clover\\clover.xml\")\n",
    "root = tree.getroot()\n",
    "root_dict = etree_to_dict(root)['coverage']\n",
    "source_packages: list = root_dict['project'][0]['package']\n",
    "test_packages: list = root_dict['testproject'][0]['package']\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-30T05:34:35.456634800Z",
     "start_time": "2023-12-30T05:34:35.317389900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"type\": \"student\"}\n"
     ]
    },
    {
     "data": {
      "text/plain": "Person(type=<Type.STUDENT: 'student'>)"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from enum import Enum\n",
    "from pydantic import BaseModel\n",
    "from typing import Optional\n",
    "\n",
    "\n",
    "class Person(BaseModel):\n",
    "    class Type(Enum):\n",
    "        STUDENT = \"student\"\n",
    "        TEACHER = \"teacher\"\n",
    "\n",
    "        def __str__(self):\n",
    "            return self.value\n",
    "\n",
    "    type: Type = Type.STUDENT\n",
    "\n",
    "\n",
    "import json\n",
    "\n",
    "obj = json.dumps(Person().model_dump(), default=str)\n",
    "print(obj)\n",
    "Person(**json.loads(obj))\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-07T06:29:00.931494300Z",
     "start_time": "2024-01-07T06:29:00.923995100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-07T06:52:34.262873600Z",
     "start_time": "2024-01-07T06:52:34.260032200Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
